<!DOCTYPE html>
<html lang="en">
  <head>
  <meta charset="utf-8" />
  <meta http-equiv="X-UA-Compatible" content="IE=edge">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">
  <meta content="" name="keywords">

  <title> BeautifulSoup - </title>
  <meta name="description" content="" />
  <meta property="og:title" content="BeautifulSoup" />
  <meta name="twitter:title" content="BeautifulSoup" />
  <meta name="author" content="{Description { .Site.Author.name }}"/>
  <meta name="twitter:card" content="summary" />
  <meta property="og:url" content="https://systemime.github.io/tags/beautifulsoup/" />
  <meta property="og:type" content="website" />
  <meta property="og:site_name" content="听风的小站" />

  <meta name="generator" content="Hugo 0.67.0" />
  <link rel="canonical" href="https://systemime.github.io/tags/beautifulsoup/" />
  <link rel="alternate" href="https://systemime.github.io/index.xml" type="application/rss+xml" title="听风的小站">
  <script src="https://res.cloudinary.com/jimmysong/raw/upload/rootsongjc-hugo/jquery-1.12.4.min.js"></script>
  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.7.1/katex.min.css" integrity="sha384-wITovz90syo1dJWVh32uuETPVEtGigN07tkttEqPv+uR2SE/mbQcG7ATL28aI9H0" crossorigin="anonymous">
  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css" />
  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
  
  
  <link rel="stylesheet" href="https://systemime.github.io/css/main.css" />
  <link rel="stylesheet" href="https://systemime.github.io/css/my.css" />
  <link rel="stylesheet" href="https://systemime.github.io/css/pace.css" />
  <link rel="stylesheet" href="https://cdn.plyr.io/3.4.7/plyr.css">
  <link rel="stylesheet" href="https://systemime.github.io/css/lightgallery.css" />
  <link rel="stylesheet" href="https://systemime.github.io/css/baguetteBox.css" />
  <link rel="stylesheet" href="https://systemime.github.io/css/search.css" />
  <link rel="stylesheet" href="https://systemime.github.io/css/reward.css" />
  <link rel="stylesheet" href="https://systemime.github.io/css/share.min.css" />
  <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Lora:400,700,400italic,700italic" />
  <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800" />
  <link rel="stylesheet" href="https://systemime.github.io/css/pygment_highlights.css" />
  <link rel="stylesheet" href="https://systemime.github.io/css/highlight.min.css" />
  <link rel="stylesheet" href="https://systemime.github.io/css/toc.css" />
  <link rel="stylesheet" href="https://systemime.github.io/css/lightbox.css" />
  <link rel="stylesheet" href="https://systemime.github.io/live2d/css/live2d.css" />
  
  <link href="https://systemime.github.io/css/video-js.css" rel="stylesheet">


<meta name="baidu-site-verification" content="g8IYR9SNLF" />
<script>
var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "https://hm.baidu.com/hm.js?cffe439e37449bb1c07ab26ab56484bb";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script>

<link rel="stylesheet" href="https://systemime.github.io/css/prism.css" />









<script src="https://systemime.github.io/js/pace.min.js" data-no-instant></script>



<script src="https://systemime.github.io/js/instantclick.min.js" data-no-instant></script>
   <script data-no-instant>
   InstantClick.on('change', function(isInitialLoad) {
     if (isInitialLoad === false) {
       if (typeof MathJax !== 'undefined') 
         MathJax.Hub.Queue(["Typeset",MathJax.Hub]);
       if (typeof prettyPrint !== 'undefined') 
         prettyPrint();
       if (typeof _hmt !== 'undefined')  
         _hmt.push(['_trackPageview', location.pathname + location.search]);
       if (typeof ga !== 'undefined')  
           ga('send', 'pageview', location.pathname + location.search);
     }
   });
   InstantClick.init();
</script>




<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/gist-embed/2.7.1/gist-embed.min.js"></script>


<script async src="//pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script>
<script>
  (adsbygoogle = window.adsbygoogle || []).push({
    google_ad_client: "ca-pub-3925981084585036",
    enable_page_level_ads: true
  });
</script>



<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/photoswipe/4.1.1/photoswipe.min.css" integrity="sha256-sCl5PUOGMLfFYctzDW3MtRib0ctyUvI9Qsmq2wXOeBY=" crossorigin="anonymous" />
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/photoswipe/4.1.1/default-skin/default-skin.min.css" integrity="sha256-BFeI1V+Vh1Rk37wswuOYn5lsTcaU96hGaI7OUVCLjPc=" crossorigin="anonymous" />



<div class="pswp" tabindex="-1" role="dialog" aria-hidden="true">

<div class="pswp__bg"></div>

<div class="pswp__scroll-wrap">
    
    <div class="pswp__container">
      <div class="pswp__item"></div>
      <div class="pswp__item"></div>
      <div class="pswp__item"></div>
    </div>
    
    <div class="pswp__ui pswp__ui--hidden">
    <div class="pswp__top-bar">
      
      <div class="pswp__counter"></div>
      <button class="pswp__button pswp__button--close" title="Close (Esc)"></button>
      <button class="pswp__button pswp__button--share" title="Share"></button>
      <button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>
      <button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>
      
      
      <div class="pswp__preloader">
        <div class="pswp__preloader__icn">
          <div class="pswp__preloader__cut">
            <div class="pswp__preloader__donut"></div>
          </div>
        </div>
      </div>
    </div>
    <div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
      <div class="pswp__share-tooltip"></div>
    </div>
    <button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)">
    </button>
    <button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)">
    </button>
    <div class="pswp__caption">
      <div class="pswp__caption__center"></div>
    </div>
    </div>
    </div>
</div>

</head>

  <body>
    <nav class="navbar navbar-default navbar-fixed-top navbar-custom">
  <div class="container-fluid">
    <div class="navbar-header">
      <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#main-navbar">
        <span class="sr-only">Toggle navigation</span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
      </button>
      <a class="navbar-brand" href="https://systemime.github.io">听风的小站</a>
    </div>
    <div class="collapse navbar-collapse" id="main-navbar">
      <ul class="nav navbar-nav navbar-right">
        
          
            <li>
		    <a title="博客" href="https://systemime.github.io/blog/">博客</a>
              
              
            </li>
          
        
          
            <li>
		    <a title="片刻" href="https://systemime.github.io/moment/">片刻</a>
              
              
            </li>
          
        
          
            <li>
		    <a title="关于" href="https://systemime.github.io/about/">关于</a>
              
              
            </li>
          
        
          
            <li>
		    <a title="留言" href="https://systemime.github.io/message/">留言</a>
              
              
            </li>
          
        

        

        
        
      </ul>
    </div>

    <div class="avatar-container">
      <div class="avatar-img-border">
        
      </div>
    </div>

  </div>
</nav>






    
  <header class="header-section ">
    <div class="intro-header no-img">
      <div class="container">
        <div class="row">
          <div class="col-lg-12 col-md-12 col-md-offset-0">
            <div class="page-heading">
              <h1>#BeautifulSoup</h1>
              <hr class="small">
            </div>
          </div>
        </div>
      </div>
    </div>
  </header>

    
  <div class="container" role="main">
    <div class="row">
      <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
        
        <div class="posts-list">
          
            <article class="post-preview">
              <a href="https://systemime.github.io/blog/2019-04/%E5%9F%BA%E4%BA%8Eseleniumbeautifulsoup%E4%B8%8Egoose3%E7%9A%84%E5%8A%A8%E6%80%81%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB%E4%BB%A5%E4%B8%9C%E6%96%B9%E8%A7%86%E9%87%8E%E4%B8%BA%E4%BE%8B/">
                <h2 class="post-title">基于selenium、BeautifulSoup与goose3的动态网络爬虫（以东方视野为例）</h2>

                
              </a>

              <p class="post-meta">
                Posted on April 27, 2019
              </p>
              <div class="post-entry">
                
                  <h2 id="爬虫所需环境">爬虫所需环境</h2>
<ul>
<li>python3.6.5</li>
<li>selenium</li>
<li>selenium对应浏览器驱动</li>
<li>BeautifulSoup</li>
<li>goose3</li>
</ul>
<h2 id="保证网速运行下面代码">保证网速运行下面代码</h2>
<div class="highlight"><pre class="chroma"><code class="language-python" data-lang="python"><span class="c1"># encoding=utf8</span>
<span class="kn">from</span> <span class="nn">selenium.webdriver.support</span> <span class="kn">import</span> <span class="n">expected_conditions</span> <span class="k">as</span> <span class="n">EC</span>
<span class="kn">from</span> <span class="nn">bs4</span> <span class="kn">import</span> <span class="n">BeautifulSoup</span>
<span class="kn">import</span> <span class="nn">time</span>
<span class="kn">import</span> <span class="nn">csv</span>
<span class="kn">from</span> <span class="nn">selenium</span> <span class="kn">import</span> <span class="n">webdriver</span>
<span class="kn">from</span> <span class="nn">goose3</span> <span class="kn">import</span> <span class="n">Goose</span>
<span class="kn">from</span> <span class="nn">goose3.text</span> <span class="kn">import</span> <span class="n">StopWordsChinese</span>  <span class="c1"># 中文库</span>
<span class="kn">import</span> <span class="nn">shutil</span><span class="o">,</span> <span class="nn">os</span>

<span class="k">def</span> <span class="nf">nChrome</span><span class="p">():</span>  <span class="c1"># 后台浏览器模式</span>
    <span class="n">chrome_options</span> <span class="o">=</span> <span class="n">webdriver</span><span class="o">.</span><span class="n">ChromeOptions</span><span class="p">()</span>
    <span class="n">chrome_options</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s1">&#39;--headless&#39;</span><span class="p">)</span>
    <span class="n">driver</span> <span class="o">=</span> <span class="n">webdriver</span><span class="o">.</span><span class="n">Chrome</span><span class="p">(</span><span class="n">chrome_options</span><span class="o">=</span><span class="n">chrome_options</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">driver</span>

<span class="c1"># 前台浏览器模式</span>
<span class="k">def</span> <span class="nf">openChrome</span><span class="p">():</span>
    <span class="k">try</span><span class="p">:</span>
        <span class="n">option</span> <span class="o">=</span> <span class="n">webdriver</span><span class="o">.</span><span class="n">ChromeOptions</span><span class="p">()</span>
        <span class="n">option</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s1">&#39;disable-infobars&#39;</span><span class="p">)</span>
        <span class="n">driver</span> <span class="o">=</span> <span class="n">webdriver</span><span class="o">.</span><span class="n">Chrome</span><span class="p">(</span><span class="n">chrome_options</span><span class="o">=</span><span class="n">option</span><span class="p">)</span>
        <span class="c1"># driver.maximize_window()  # 最大化浏览器</span>
        <span class="k">return</span> <span class="n">driver</span>
    <span class="k">except</span><span class="p">:</span>
        <span class="k">print</span><span class="p">(</span><span class="s2">&#34;有错误发生，驱动程序缺失或浏览器版本不匹配，或环境变量未配置&#34;</span><span class="p">)</span>
        <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
        <span class="nb">exit</span><span class="p">()</span>

<span class="c1"># 爬虫主体</span>
<span class="k">def</span> <span class="nf">Open_w</span><span class="p">(</span><span class="n">w_num</span><span class="p">):</span>
    <span class="n">base_url</span> <span class="o">=</span> <span class="s2">&#34;http://finance.eastmoney.com/a/cdfsd_{page}.html&#34;</span>
    <span class="n">num</span> <span class="o">=</span> <span class="mi">1</span>  <span class="c1"># 文章计数</span>
    <span class="n">page</span> <span class="o">=</span> <span class="mi">1</span>  <span class="c1"># 页数计数</span>
    <span class="n">entry</span> <span class="o">=</span> <span class="mi">1</span>  <span class="c1"># 条目计数</span>
    <span class="n">articles</span> <span class="o">=</span> <span class="p">[]</span>  <span class="c1"># csv内容存入</span>
    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">w_num</span><span class="o">+</span><span class="mi">1</span><span class="p">):</span>  <span class="c1"># 按页爬取</span>
        <span class="n">url</span> <span class="o">=</span> <span class="n">base_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">page</span><span class="o">=</span><span class="n">i</span><span class="p">)</span>  <span class="c1"># 按照网页链接规律</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">driver</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>  <span class="c1"># 打开网页</span>
            <span class="c1"># lxml解析器 速度快 文档纠错能力强 需要安装C语言库</span>
            <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>  <span class="c1"># 缓冲</span>
            <span class="n">titles</span> <span class="o">=</span> <span class="n">BeautifulSoup</span><span class="p">(</span><span class="n">driver</span><span class="o">.</span><span class="n">page_source</span><span class="p">,</span> <span class="s1">&#39;lxml&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">&#39;div&#39;</span><span class="p">,</span> <span class="p">{</span><span class="s1">&#39;class&#39;</span><span class="p">:</span> <span class="s1">&#39;text&#39;</span><span class="p">})</span>  <span class="c1"># 解析div部分代码</span>
            <span class="k">if</span> <span class="n">titles</span><span class="p">:</span>
                <span class="k">print</span><span class="p">(</span><span class="s2">&#34;||||||------------第{0}页爬取内容已获得------------||||||&#34;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">page</span><span class="p">))</span>
                <span class="k">for</span> <span class="n">title</span> <span class="ow">in</span> <span class="n">titles</span><span class="p">:</span>  <span class="c1"># 注意，xpath失败由于类型错误</span>
                    <span class="nb">id</span> <span class="o">=</span> <span class="n">num</span>
                    <span class="n">new_title</span> <span class="o">=</span> <span class="n">title</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;a&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39; &#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>  <span class="c1"># 获取a标签内容并去除空格，回车</span>
                    <span class="n">new_summ</span> <span class="o">=</span> <span class="n">title</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;p&#39;</span><span class="p">,</span> <span class="p">{</span><span class="s1">&#39;class&#39;</span><span class="p">:</span> <span class="s1">&#39;info&#39;</span><span class="p">})</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39; &#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
                    <span class="n">new_time</span> <span class="o">=</span> <span class="n">title</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;p&#39;</span><span class="p">,</span> <span class="p">{</span><span class="s1">&#39;class&#39;</span><span class="p">:</span> <span class="s1">&#39;time&#39;</span><span class="p">})</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39; &#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
                    <span class="n">url</span> <span class="o">=</span> <span class="n">title</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;a&#39;</span><span class="p">)[</span><span class="s1">&#39;href&#39;</span><span class="p">]</span>  <span class="c1"># 获取a标签href内容</span>
                    <span class="c1"># sub_ord(title, url)  # 提速开进程4</span>
                    <span class="k">if</span> <span class="n">num</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
                        <span class="n">make_path</span><span class="p">()</span>  <span class="c1"># 判断储存文件夹是否存在</span>
                    <span class="n">Identification</span><span class="p">(</span><span class="n">new_title</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>  <span class="c1"># 爬取每页文章内容</span>
                    <span class="n">articles</span><span class="o">.</span><span class="n">append</span><span class="p">([</span><span class="nb">id</span><span class="p">,</span> <span class="n">new_title</span><span class="p">,</span> <span class="n">new_summ</span><span class="p">,</span> <span class="n">new_time</span><span class="p">,</span> <span class="n">url</span><span class="p">])</span>  <span class="c1"># 添加内容</span>
                    <span class="n">num</span> <span class="o">=</span> <span class="n">num</span> <span class="o">+</span> <span class="mi">1</span>
                    <span class="n">entry</span> <span class="o">=</span> <span class="n">entry</span> <span class="o">+</span> <span class="mi">1</span>
                <span class="n">entry</span> <span class="o">=</span> <span class="mi">1</span>
                <span class="k">print</span><span class="p">(</span><span class="s2">&#34;||||||------------第{0}页爬取完毕！！！------------||||||&#34;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">page</span><span class="p">))</span>
                <span class="n">page</span> <span class="o">=</span> <span class="n">page</span> <span class="o">+</span> <span class="mi">1</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="k">print</span><span class="p">(</span><span class="s2">&#34;请检查网页结构是否发生变化&#34;</span><span class="p">)</span>
                <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
                <span class="nb">exit</span><span class="p">()</span>
        <span class="k">except</span><span class="p">:</span>
            <span class="k">print</span><span class="p">(</span><span class="s2">&#34;请更换ip或增加代理池，预防黑名单&#34;</span><span class="p">)</span>
            <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
            <span class="nb">exit</span><span class="p">()</span>
    <span class="n">save_data</span><span class="p">(</span><span class="n">articles</span><span class="p">)</span>  <span class="c1"># 存储文件</span>

<span class="c1"># 识别模式</span>
<span class="k">def</span> <span class="nf">Identification</span><span class="p">(</span><span class="n">title</span><span class="p">,</span> <span class="n">url</span><span class="p">):</span>
    <span class="n">g</span> <span class="o">=</span> <span class="n">Goose</span><span class="p">({</span><span class="s1">&#39;stopwords_class&#39;</span><span class="p">:</span> <span class="n">StopWordsChinese</span><span class="p">})</span>  <span class="c1"># 使用中文包</span>
    <span class="n">article</span> <span class="o">=</span> <span class="n">g</span><span class="o">.</span><span class="n">extract</span><span class="p">(</span><span class="n">url</span><span class="o">=</span><span class="n">url</span><span class="p">)</span>  <span class="c1"># 识别url</span>
    <span class="k">try</span><span class="p">:</span>
        <span class="n">filename</span> <span class="o">=</span> <span class="s2">&#34;.</span><span class="se">\\</span><span class="s2">Content</span><span class="se">\\</span><span class="s2">&#34;</span> <span class="o">+</span> <span class="n">title</span> <span class="o">+</span> <span class="s2">&#34;.txt&#34;</span>  <span class="c1"># 存储路径与名称</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s1">&#39;w&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
            <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">article</span><span class="o">.</span><span class="n">cleaned_text</span><span class="p">[</span><span class="mi">0</span><span class="p">:])</span>  <span class="c1"># 写入内容</span>
    <span class="k">except</span> <span class="ne">IOError</span><span class="p">:</span>
        <span class="k">print</span><span class="p">(</span><span class="s2">&#34;文件发生异常，请检查文件！！！&#34;</span><span class="p">)</span>
        <span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
        <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
        <span class="nb">exit</span><span class="p">()</span>

<span class="k">def</span> <span class="nf">make_path</span><span class="p">():</span>
    <span class="k">if</span> <span class="s2">&#34;Content&#34;</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">():</span>
        <span class="k">if</span> <span class="nb">input</span><span class="p">(</span><span class="s2">&#34;当前目录下存储文件夹Content已存在，是否需要删除？Y/N:   &#34;</span><span class="p">)</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;Y&#39;</span><span class="p">,</span> <span class="s1">&#39;y&#39;</span><span class="p">]:</span>
            <span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="s2">&#34;Content&#34;</span><span class="p">)</span>
            <span class="n">os</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="s2">&#34;Content&#34;</span><span class="p">)</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">os</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="s2">&#34;Content&#34;</span><span class="p">)</span>

<span class="c1"># 普通模式, 速度太慢！！！</span>
<span class="c1"># def sub_ord(sub_title, url):</span>
<span class="c1">#     driver.get(url)</span>
<span class="c1">#     titles = BeautifulSoup(driver.page_source, &#39;lxml&#39;).find_all(&#39;div&#39;, {&#39;class&#39;: &#39;Body&#39;})</span>
<span class="c1">#     if titles:</span>
<span class="c1">#         for title in titles:</span>
<span class="c1">#             new_summ = title.get_text().replace(&#39; &#39;, &#39;&#39;).strip(&#39;\n&#39;)</span>
<span class="c1">#             print(new_summ)</span>
<span class="c1">#     else:</span>
<span class="c1">#         print(&#34;请检查网页结构是否发生变化&#34;)</span>
<span class="c1">#         time.sleep(5)</span>
<span class="c1">#         exit()</span>
<span class="c1">#     try:</span>
<span class="c1">#         filename = sub_title + &#34;.txt&#34;</span>
<span class="c1">#         with open(filename, &#39;w&#39;) as f:</span>
<span class="c1">#             f.write(new_summ)</span>
<span class="c1">#     except IOError:</span>
<span class="c1">#         print(&#34;文件发生异常，请检查文件！！！&#34;)</span>
<span class="c1">#         f.close()</span>
<span class="c1">#         time.sleep(5)</span>
<span class="c1">#         exit()</span>
<span class="c1">#     driver.close()</span>

<span class="c1"># def Content(driver):</span>
<span class="c1">#         try:</span>
<span class="c1">#             links = driver.find_elements_by_xpath(&#34;//*[@id=&#39;newsListContent&#39;]/li[&#34; + str(i) + &#34;]/div[2]/p[1]/a&#34;)</span>
<span class="c1">#             for link in links:</span>
<span class="c1">#                 urls.append(link.get_attribute(&#39;href&#39;))</span>
<span class="c1">#         except:</span>
<span class="c1">#             print(&#34;写法出现错误&#34;)</span>

<span class="k">def</span> <span class="nf">save_data</span><span class="p">(</span><span class="n">articles</span><span class="p">):</span>
    <span class="k">try</span><span class="p">:</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s1">&#39;save.csv&#39;</span><span class="p">,</span> <span class="s1">&#39;w&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
            <span class="n">writer</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">writer</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
            <span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">([</span><span class="s1">&#39;文章id&#39;</span><span class="p">,</span> <span class="s1">&#39;文章标题&#39;</span><span class="p">,</span> <span class="s1">&#39;文章摘要&#39;</span><span class="p">,</span> <span class="s1">&#39;发表时间&#39;</span><span class="p">,</span> <span class="s1">&#39;文章链接&#39;</span><span class="p">])</span>
            <span class="k">try</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">articles</span><span class="p">:</span>
                    <span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">(</span><span class="n">row</span><span class="p">)</span>
                <span class="k">print</span><span class="p">(</span><span class="s2">&#34;保存完毕！！！随时停止&#34;</span><span class="p">)</span>
            <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
                <span class="k">print</span><span class="p">(</span><span class="s2">&#34;文件写入发生异常，请检查数据！！！&#34;</span><span class="p">)</span>
                <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
                <span class="nb">exit</span><span class="p">()</span>
    <span class="k">except</span> <span class="ne">IOError</span><span class="p">:</span>
        <span class="k">print</span><span class="p">(</span><span class="s2">&#34;文件发生异常，请检查文件！！！&#34;</span><span class="p">)</span>
        <span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
        <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
        <span class="nb">exit</span><span class="p">()</span>

<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">&#39;__main__&#39;</span><span class="p">:</span>
    <span class="c1"># w_num = int(input(&#34;请输入爬取页面数量: &#34;))</span>
    <span class="n">w_num</span> <span class="o">=</span> <span class="mi">2</span>
    <span class="n">driver</span> <span class="o">=</span> <span class="n">nChrome</span><span class="p">()</span>
    <span class="k">while</span> <span class="mi">1</span><span class="p">:</span>
        <span class="n">result</span> <span class="o">=</span> <span class="n">EC</span><span class="o">.</span><span class="n">alert_is_present</span><span class="p">()(</span><span class="n">driver</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">result</span><span class="p">:</span>
            <span class="k">print</span><span class="p">(</span><span class="s2">&#34;alert 存在弹窗，处理后再试验&#34;</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">print</span><span class="p">(</span><span class="s2">&#34;alert 未弹出！&#34;</span><span class="p">)</span>
            <span class="k">break</span>
    <span class="n">Open_w</span><span class="p">(</span><span class="n">w_num</span><span class="p">)</span>
    <span class="c1"># content = driver.page_source.encode(&#39;utf-8&#39;)</span>
    <span class="n">confirm</span> <span class="o">=</span> <span class="nb">input</span><span class="p">(</span><span class="s2">&#34;数据已成功爬取并存储，是否查看？Y（查看文件）|N（关闭进程）：&#34;</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">confirm</span> <span class="o">==</span> <span class="s1">&#39;Y&#39;</span> <span class="ow">or</span> <span class="n">confirm</span> <span class="o">==</span> <span class="s1">&#39;y&#39;</span><span class="p">:</span>
        <span class="kn">import</span> <span class="nn">os</span>
        <span class="n">os</span><span class="o">.</span><span class="n">system</span><span class="p">(</span><span class="s2">&#34;start save.csv&#34;</span><span class="p">)</span>
        <span class="n">driver</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">driver</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
</code></pre></div>
                
              </div>

              
                <span class="post-meta">
                  
                    #<a href="https://systemime.github.iotags/%E7%88%AC%E8%99%AB/">爬虫</a>&nbsp;
                  
                    #<a href="https://systemime.github.iotags/selenium/">selenium</a>&nbsp;
                  
                    #<a href="https://systemime.github.iotags/beautifulsoup/">BeautifulSoup</a>&nbsp;
                  
                    #<a href="https://systemime.github.iotags/goose3/">goose3</a>&nbsp;
                  
                </span>
              

            </article>
          
        </div>
        
      </div>
    <div align="center" class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
        


        
        
      </div>
    </div>
  </div>

    <div class="rocket"><img src="https://systemime.github.io/img/rocket_up.png" alt="" width="100" height="100"></div>
<script>
    $(function () {
      $(window).scroll(function () {
        
        if ($(window).scrollTop() >= 1000) {
          $('.rocket').stop().fadeIn(1000);
        }else {
          $('.rocket').stop().fadeOut(1000);
        }
      })
      
      $('.rocket').click(function () {
        $('html,body').stop().animate({scrollTop:0},400);
       
      })
    })

</script>

    
    <div id="landlord">
        <div class="message" style="opacity:0"></div>
        <canvas id="live2d" width="280" height="250" class="live2d"></canvas>
        <div class="hide-button">隐藏</div>
    </div>
    
    <footer id="copyright">
  <div class="container">
    <div class="row">
      <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
        <ul class="list-inline text-center footer-links">
          
          
        </ul>
        <p class="credits copyright text-muted">
        &copy;2017-2018
          
          &nbsp;&bull;&nbsp;
          September 20,2019
          updated
          
            &nbsp;&bull;&nbsp;
            <a href="https://systemime.github.io">Home</a>
          
        </p>

        <p class="credits theme-by text-muted">
        <a href="https://servicemesher.github.io" target="_blank">ServiceMesher</a>&nbsp;&bull;&nbsp;<a href="https://github.com/yangchuansheng/k8s-knowledge" target="_blank">Kubernetes 知识图谱</a>
        <br />
        <span id="busuanzi_container_site_pv">
            本站已被访问 <span id="busuanzi_value_site_pv"></span> 次啦
        </span>
        <span id="busuanzi_container_site_uv">
            &ensp;|&ensp;&thinsp;您是第 <span id="busuanzi_value_site_uv"></span> 位到访的小伙伴喔
        </span>
        </p>
        
        <p class="credits theme-by text-muted">
          <a href="http://gohugo.io">Hugo v0.67.0</a> powered &nbsp;&bull;&nbsp; Theme by <a href="http://deanattali.com/beautiful-jekyll/">Beautiful Jekyll</a> adapted to <a href="https://github.com/halogenica/beautifulhugo">Beautiful Hugo</a>
          
          <center><section class="credits theme-by text-muted">
    <span class="footer__copyright">
    <div>
    <span id="span_dt_dt"> </span>
    <script language="javascript">
      function show_date_time(){
        window.setTimeout("show_date_time()", 1000);
        BirthDay=new Date("7/8/2017 10:56:12");
        today=new Date();
        timeold=(today.getTime()-BirthDay.getTime());
        sectimeold=timeold/1000
        secondsold=Math.floor(sectimeold);
        msPerDay=24*60*60*1000
        e_daysold=timeold/msPerDay
        daysold=Math.floor(e_daysold);
        e_hrsold=(e_daysold-daysold)*24;
        hrsold=Math.floor(e_hrsold);
        e_minsold=(e_hrsold-hrsold)*60;
        minsold=Math.floor((e_hrsold-hrsold)*60);
        seconds=Math.floor((e_minsold-minsold)*60);
        span_dt_dt.innerHTML="本博客已经开心运行 "+daysold+" 天 "+hrsold+" 小时 "+minsold+" 分 "+seconds+" 秒";
      }
      show_date_time();
    </script>
    </div>
</script>
</section>
</center>
        </p>
      </div>
    </div>
  </div>
</footer>


<script src="https://res.cloudinary.com/jimmysong/raw/upload/rootsongjc-hugo/bootstrap.min.js"></script>
<script src="https://res.cloudinary.com/jimmysong/raw/upload/rootsongjc-hugo/photoswipe-ui-default.min.js"></script>
<script src="https://res.cloudinary.com/jimmysong/raw/upload/rootsongjc-hugo/photoswipe.min.js"></script>
<script src="https://res.cloudinary.com/jimmysong/raw/upload/rootsongjc-hugo/auto-render.min.js"></script>
<script src="https://systemime.github.io/js/main.js"></script>
<script src="https://systemime.github.io/js/clipboard.min.js"></script>
<script src="https://systemime.github.io/js/prism.js?t=123"></script>
<script src="https://systemime.github.io/js/highlight.min.js"></script>
<script src="https://res.cloudinary.com/jimmysong/raw/upload/rootsongjc-hugo/katex.min.js"></script>
<script src="https://systemime.github.io/js/reward.js"></script>
<script src="https://systemime.github.io/js/canvas_ribbon.js"></script>
<script> renderMathInElement(document.body); </script>

<script src="https://systemime.github.io/js/baguetteBox.js"></script>
<script> baguetteBox.run('.gallery');</script>






<script async defer src="https://buttons.github.io/buttons.js"></script>


<script src="https://polyfill.io/v2/polyfill.min.js?features=IntersectionObserver"></script>
<script type="text/javascript" src="https://systemime.github.io/js/quicklink.umd.js"></script>
<script>
window.addEventListener('load', () =>{
   quicklink();
});
</script>



<script type="text/javascript">
 
var a_idx = 0;
jQuery(document).ready(function($) {
    $("body").click(function(e) {
        var a = new Array("Docker", "Kubernetes", "Prometheus", "Envoy", "Istio", "Service Mesh", "Cloud Native");
        var $i = $("<span />").text(a[a_idx]);
        a_idx = (a_idx + 1) % a.length;
        var x = e.pageX,
        y = e.pageY;
        function randomColor() {
          var flakeColor = new Array("#FFDA65", "#00BFFF", "#BA55D3", "#FFA07A", "#87CEEB", "#FFB6C1");
          var snowColor = flakeColor[Math.floor(flakeColor.length * Math.random())];
          return snowColor;
        }
        $i.css({
            "z-index": 999999999999999999999999999999999999999999999999999999999999999999999,
            "top": y - 20,
            "left": x,
            "position": "absolute",
            "font-weight": "bold",
            "color": randomColor()
        });
        $("body").append($i);
        $i.animate({
            "top": y - 180,
            "opacity": 0
        },
        1500,
        function() {
            $i.remove();
        });
    });
});
</script>


<script type="text/javascript" src="https://systemime.github.io/js/lightbox.js"></script>











<script src="https://cdn.plyr.io/3.4.7/plyr.js"></script>
<script>const player = new Plyr('#player');</script>


<script async defer src="https://buttons.github.io/buttons.js"></script>


  </body>
</html>

